import asyncio

from parsel import Selector

from re_common.baselibrary.utils.basemotor import BaseMotor

bs = BaseMotor()
bs.AsyncIOMotorClient(
    "mongodb://cjrw:vipdatacenter@192.168.31.243:32920,192.168.31.208:32920,192.168.31.206:32920/testdb.jss_patent_page?authSource=testdb",
    "testdb")
bs.get_col("jss_patent_page")


async def dic_hook(doc):
    print(doc["_id"])
    htmls = doc["html"]
    html = Selector(htmls)
    ul_list = html.xpath("//ul[@class='list_ul']")
    lists = []
    for ul in ul_list:
        detail_url_list = ul.xpath("./li[@class='title_li']/a[@class='abs_img']/@href").extract()
        detail_url = detail_url_list[0].replace("_free", "")
        rawid = detail_url.replace("http://sjk15.e-library.com.cn/D/", "").replace(".aspx", "")
        lists.append(rawid)
    lists.sort()
    rawid_strings = ";".join(lists)
    num = len(lists)
    bs.update({"_id": doc["_id"]}, {"rawidstr": rawid_strings, "rawidnum": num})


async def run():
    await bs.find(dic_hook, {"rawidnum": {"$exists": False}}, ['_id', 'html'])


asyncio.get_event_loop().run_until_complete(run())
